# -*- coding: utf-8 -*-

#    Stingingnettlepy - A static website generator written in Python
#    Copyright (C) 2013  Jannik Haberbosch
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.


# Importion of modules from the Python Standard Library.
import sys
import string
import os
import json
import shutil
import ast
import copy
import importlib
import re
import logging
import socket
import posixpath
import filecmp
import collections
import SimpleHTTPServer
import SocketServer
import time
import ftputil

# Importion of 3rd party modules.
import paramiko

# Importion of own modules and code-fragments of own modules.
from stingingnettlepy.other.stingingnettlepy_constants import *
from stingingnettlepy.other.functions import *


class StingingNettlePy( object ):
	"""Main Class to generate a static website, including copying it to a local test-server and uploading it to a remote server.

	The 'StingingNettlePy' class generates the static HTML website by using everything
	that is inside 'in/', aswell as some other directories and files defined in the root
	directory 'SWPD', and puts the generated website into the 'out/' directory.
	Additionally it can be used to either copy the website to a specific directory on
	another location on the local computer, or upload it to a remote Linux server.

	Let us review an example in order to see which directories and files are takem from
	the 'SPWD' directory to generate the static website:

	It is assumed that our 'SWPD' directory tree looks like this:
		SWPD/
			in/
				_css/
					basic.css
				_js/
					basic.js
				_media/
					_photos/
						img1.JPG
						img2.JPG
						img3.JPG
					_music/
						track1.mp3
						track2.flac
					_sounds/
						sound1.wav
						sound2.wav
						sound3.wav
					_videos/
						video1.avi
						video2.avi
				0_home/
					0_index.txt
					local.config
				1_about/
					0_index.txt
					local.config
				2_diary/
					0_index.txt
					1_january_2014.txt
					2_february_2014.txt
				global.config
			out/		
			templates/
				basic.template
			modules/
				__init__.py
				mod_navigation.py
			stingingnettlepy.config

	To generate the static HTML website, the following command is called from the Terminal
	or command-line from within the 'SWPD' directory:

		stingingnettlepy-make.py

	And that command would, if there are no mistakes, generate the static website for us,
	and put the generated website into the 'SWPD/out/' directory.  So this is how the 'SWPD'
	directory structure would look like after generating the static website:
	
		SWPD/
			in/
				_css/
					basic.css
				_js/
					basic.js
				_media/
					_photos/
						img1.JPG
						img2.JPG
						img3.JPG
					_music/
						track1.mp3
						track2.flac
					_sounds/
						sound1.wav
						sound2.wav
						sound3.wav
					_videos/
						video1.avi
						video2.avi
				0_home/
					0_index.txt
					local.config
				1_about/
					0_index.txt
					local.config
				2_diary/
					0_index.txt
					1_january_2014.txt
					2_february_2014.txt
				global.config
			out/
				_css/
					basic.css
				_js/
					basic.js
				_media/
					_photos/
						img1.JPG
						img2.JPG
						img3.JPG
					_music/
						track1.mp3
						track2.flac
					_sounds/
						sound1.wav
						sound2.wav
						sound3.wav
					_videos/
						video1.avi
						video2.avi
				home/
					index.html
				about/
					index.html
				diary/
					index.html
					january_2014.html
					february_2014.html			
			templates/
				basic.template
			modules/
				__init__.py
				mod_navigation.py
			stingingnettlepy.config	

	If we were to use the following command:

		stingingnettlepy-make.py -t, --test

	it would aswell yield the same 'SWPD' directory as above, but also copy
	the contents of 'SWPD/out/' to another destination on the local computer.
	This other 'destination' actually is another directory, whose basename is
	actually taken from what has been set inside the 'stingingnettlepy.config'
	file.  More information about this later.

	To generate a static HTML website, the following files and directories have
	to exist in the 'SWPD' tree: (NOTE: This is not true if one decides to use
	the -t, --test or -u, --upload parameters.)

		'SWPD/in/'
		'SWPD/modules/'
		'SWPD/templates/'

		'SWPD/templates/basic.template'
		'SWPD/modules/__init__.py'
		'SWPD/in/global.config'

	A directory inside the 'SWPD/in/' directory tree, that starts with an '_'
	(underscore) character is ignored when generating the website.  In short,
	this directory will simply be copied to the 'SWPD/out/...' directory.
	Files that are inside such a directory won't be changed either, and will
	also be copied to the 'SWPD/out/...' directory.
	That is not true for sub-directories that do not start with an '_'!

	I'll now describe the directories and files in the first 'SWPD' tree mentioned
	above, and explain, what they are for, how their syntax has to look, why they
	have to have such ackward names, and what happens to them when the static
	HTML website is generated.


	in
	==
	Contains almost all files that are taken if a static HTML website is being created.

	in/_css/
	========
	This directory is suited to contain Cascading Style Sheets.

	in/_js/
	=======
	This directory is suited to contain JavaScript files.

	in/_media/
	==========
	This directory is used to include several other media files.

	in/_media/_photos/
	==================
	This directory should be used to include graphics and photos in a website.
	Nothing special needs to be said about the files 'img1.JPG', 'img2.JPG'
	and 'img3.JPG'; they are just regular image files.
	They won't be altered in any way.

	in/_media/_music/
	=================
	This directory should be used to include music in a website.
	Nothing special needs to be said about the files 'track1.mp3' and 'track2.flac'
	they are just regular music files.
	They won't be altered in any way.

	in/_media/_sounds/
	==================
	This directory should be used to include sounds in a website.
	Nothing special needs to be said about the files 'sound1.wav', 'sound2.wav'
	and 'sound3.wav'; they are just regular .wav files.
	They won't be altered in any way.	

	in/_media/_videos/
	==================
	This directory should be used to include videos in a website.
	Nothing special needs to be said about the files 'video1.avi' and 'video1.avi';
	they are just regular video files.
	They won't be altered in any way.

	The website directories '0_home', '1_about', 2_diary'
	=====================================================
	These directories, easily put, define sub-sections of a website.  A sub directory
	has to begin with a sequence of one or more digits (1,2,3,...) and followed by an '_'
	(underscore), followed by the name of the sub-section.  The name of the sub-section
	may then be alphanumeric, including additional '_' (underscores).

	The digits help sorting a navigation in a specific way when the website is being
	created.  The user name later define, by the use of modules, if sub-directories with
	a lower index number should be included further at the start of the navigation of
	the website, or the other way round.  Usually, it is best to say that the lower the
	index number, the further it is at the start of a navigation.

	After the website has been generated, the leading sequence of digits and the first
	'_' (underscore) are removed from the directory, thus renaming the directory.
	

	The user has the ability to include directories that start with an '_'.  As mentioned
	before, these are simply ignored and won't be altered in any way if the static HTML
	website is being created.

	The files '0_index.txt', '0_index.txt', '0_index.txt', '1_january_2014.txt' and
	'2_february_2014.txt' ('TTH' files)
	=================================================================================
	Each of these files represents a page of a static website.  Thus, I will from now on
	refer to such a file by using the abbreviation 'TTH', which stands for 'Text To HTML'.

	A TTH file is a file that will, when the website is being generated, be taken and run
	through a template, of which the result is a new file, a HTML file.

	Naming:
		A TTH file's name is as important as is the name of a website sub-sections.  They
		have to begin with a sequence of one or more digits (1, 2, 3, ...).  After this
		sequence, an '_' (underscore) has to follow.  After this underscore, any sequence
		of alpahnumeric characters, including other '_' may follow.

		The use of this naming techniques is the same as it is for the directories of a
		website:  To sort files for a sub-navigation on a website.

		After the website has been generated, the leading sequence of digits and the first
		'_' (underscore) are removed from the file, thus renaming the file.

	Source:
		The source of a TTH file is usually made up of two sections, whereas the first section
		may be omitted.
		
		The first section is called the 'file_config' section.
		The second section is called the 'containers' section.

		Here is an example of how a file with a first and a second section has to look like:
			{
				"page_heading": "DIARY",
				"mod_subnav1": {
					"module_name": "navigation.py",
					"function_name": "make_subnav",
					"parameters": [ "param1", "param2" ],
					"container_name": "subnav"
				},
				"mod_gallery1": {
					"module_name": "gallery.py",
					"function_name": "make_gallery",
					"parameters": [ "path1", ],
					"container_name": "gallery1"
				}
			}
			---config---
			|||main_content
			<p>This is just an example of what is possible.</p>
			<p>Here are a few images, showing me at my grandparents house:</p>
			$gallery1
			|||main_content
			|||other_content_container
			<p>The other placeholder $$subnav is used inside the template, which has been defined
			for this TTH file through a global or local configuration file..</p>
			|||other_content_container

		In summary, through either the global, the local or the file configuration, the user can define
		placeholders and their values.
		There is a special JSON key:value pair for functions that should be executed before a page is
		rendered.  They all have to begin with a "mod_".  Each of them contain a sub-dictionary with 4
		key:value pairs:
		
			module_name - 		The name of the file in the "modules/" directory.  It has to be a Python file.
			function_name - 	The name of the function/method in the module "module_name" that should be called.
			parameters - 		A list that contains the parameters to pass to the function "function_name".
			container_name - 	Contains the "placeholder_name" (key), whose value is the return value of the function
								"function_name".

			Actually, "container_name" should be renamed to "placeholder_name".

		The 'file configuration' overwrites any of the key:value pairs that have been defined through the 'global.config'
		or the 'local.config' file.  the remaining key:value pairs are simply added.

	The 'local.config' file(s)
	==========================
			
	The 'local.config' file(s) contain and overwrite any of the key:value pairs that have been defined in
	the 'global.config' file.  The remaining key:value pairs are simply added.
	It is used to define constants for all files in a sub-directory/sub-sections.

	It is optional.

	The structure of that file is the same as in the 'file configuration', e.g. in valid JSON.


	The 'global.config' file
	========================

	The 'global.config' file contains key:value pairs that are used throughout the entire website; for each and every
	single TTH file.

	It is mandatory.

	There is one key:value pair that has to exist:
	
		"template": "name_of_template.template"

	If that key:value pair does not exist, the static website won't be generated.

	The structure of that file is the same as in the 'file configuration', e.g. in valid JSON.	

	The directory 'out/'
	====================

	The entire generated website is stored in this directory everytime when 'stingingnettlepy-make.py' has been called.
	There won't be any backups.

	The directory 'templates/'
	==========================

	This directory contains all templates that should be used for this particular website.


	The file 'templates/basic.template'
	===================================

	This is the standard HTML template that should be used as a base for every TTH file, thus, it has to be defined in the
	'global.config' file, such as:

		{
			...,
			"template": "basic.template",
			...,
		}

	As we know, the user can still overwrite the template to use by specifing another template either in the 'local.config'
	file, or in the 'file configuration'.

	The template consists of HTML code mixed with placeholders.  A placeholder is defined by a '$' (dollar) and an alphanumeric
	sequence, including '_' (underscore).  It musn't begin with a digit.

	Here is an example:
		<!DOCTYPE ...>
		<html>
			<head>
				<title>$website_title</title>
				<meta http-equiv="content-type" content="text/html;charset=utf-8" />
			</head>
			<body>
				<div id="wrapper">
					<div id="header">
					</div>
					<div id="main_nav">
						$mainnav
					</div>
					<div id="sub_nav">
						$subnav
					</div>
					<div id="content">
						$main_content1
						$main_content2
					</div>
					<div id="footer">
						$footer
					</div>
				</div>
			</body>
		</html>

	The following placeholders have been defined:
		$website_title
		$mainnav
		$subnav
		$main_content1
		$main_content2
		$footer

	When a TTH file is being rendered to a valid HTML file, all contents will be replaced by the 'value' of the matching 'key'.
	These are defined either through the 'global.config' file, the 'local.config' file, a TTH file's 'file configuraton' or
	'placeholder containers'.

	In the source code, the function 'String.Template.safe_substitute()' is used, which means that there won't be an error if
	there is no matching key:value pair for a 'placeholder'.  The 'placeholder' will simply be outputted aswell.

	The user may define as many templates as he wishes to.

	The 'modules/' directory
	========================

	Contains any Python modules that should be used when generating the static website.

	The 'modules/__init__.py' file
	==============================

	This file has to exist in order for 'Stingingnettlepy' to find all modules for a website project.

	The 'modules/mod_navigation.py' file
	====================================

	This is a Python module from which a function or functions (or classes, variables, ...) should be used when generating the
	static website (a single HTML file).

	For more information on how to use modules and functions, please refer to the 'TTH files' section.

	A module may look like this:

		function concat_names( name1, name2 ):
			return name1 + name2

		function greet_user( username ):
			return "Hello %s.  Nice to have you onboard." % username

	The 'stingingnettlepy.config' file
	==================================

	This file contains configuration for 'stingingnettlepy' as soon as the script 'stingingnettlepy-make.py' is called with
	additional parameters.

	It is to be written in JSON.

	It contains the configuration for uploading the website to a distant Linux server, and for testing a website.  The file
	has to look like this:

		{
			"test": {
				"absolute_path": "...",
				"website_root_dir": "..."
			},
			"upload": {
				"username": "...",
				"password": "...",
				"absolute_path": "...",
				"website_root_dir": "...",
				"host": "...",
				"port": "...",
					
			}
		}

	Inside 'test', 'absolute_path' should contain the absolute directory path where the website should be put for testing.  This
	can be an XAMPP server.
	'website_root_dir' is the directory name of where the entire 'out/' directory should be put.

	Inside 'upload' the following keys are defined:

		name of key			explanation
		===============================
		username			The username of the user on the Linux server.
		password			The password of the use on the Linux server.
		absolute_path		The absolute path of the website-serving directory on the Linux server.
		website_root_dir	The directory name of where the website should be put in 'absolute_path'.
		host				The IP address of the Linux server.
		port				The port of the Linux server.

	For more information about uploading the website, please refer to the 'stingingnettlepy-make.py' file, and its inside
	source code documentation.

	So how is the website generated then?  What are the single steps?  How is it uploaded?
	======================================================================================

	One can find more information about that in the functions 'generate(), upload() and test()' of the
	'Stingingnettlepy' Python class.


	And this concludes the documentation.
	
	"""

	def __init__( self ):
		"""Initializes the object.
		"""

		# Load the data for the website project.
		self.website_data = load_json( STINGINGNETTLEPY_CONFIG_FILE )
		logging.debug( "Loaded website data." )

		# Store the name of the output directory of the website.
		self.website_name = self.website_data["website_name"]
		logging.debug( "Loaded the website name." )


	def test( self ):
		"""Generates the website and starts a build-in webserver.
		
		"""
		port = 8000	

		os.chdir( "out" )

		http_server_handler = SimpleHTTPServer.SimpleHTTPRequestHandler
		http_server = SocketServer.TCPServer( ( "", port ), http_server_handler )

		print "Serving your website at the following address: http://localhost:%s" % port
		print "To stop the test-server, press ctrl+c."
		http_server.serve_forever()


	def upload( self ):
		"""Generates the website and uploads it to a Linux-based web-server.
		
		This function generates the website and uploads it to a web-server.

		For more information, please read through the comments and the source-code of the function body.
		
		"""

		# Make sure that the user has provided the required login data to connect to a remote FTP Server using the FTPS protocol.
		# Check if the user has uploaded the website before.  This is done by checking if the "last_version" directory already exists.
			# If not, then upload the entire website.
			# After uploading has finished, copy the entire contents of the "out/" directory into the "last_version" directory.
			# Done.
		# Otherwise, generate hashes for the directories and files for both "last_version/" and "out/" directories.
		# Decide what has to be updated, removed or added.
		# Upload the required files and directories; delete the required files and directories; update the required files and directories.
		# After the update has been finished, delete the entire "last_version" directory, and copy the entire content of the "out/" directory
		# to the "last_version" directory.


		#####################################################################
		#
		# Make sure that the user has provided the required login data to
		# connect to a remote FTP Server using the FTPS protocol.
		#
		#####################################################################
				
		# The file 'stingingnettlepy.config' has to exist in the 'Stingingnettlepy Website Project Directory'.
		if not os.path.exists( os.path.join( os.getcwd(), STINGINGNETTLEPY_CONFIG_FILE ) ):
			#logger.error( "The file '%s' does not exist in '%s'." % ( STINGINGNETTLEPY_CONFIG_FILE, os.getcwd() ) )
			return False

		# Load the configuration that is stored in there in JSON and store it in a Python 'Dict'.
		try:
			stingingnettlepy_config = load_json( os.path.join( os.getcwd(), STINGINGNETTLEPY_CONFIG_FILE ) )
		except Exception, e:
			#logger.error( "The stingingnettlepy configuration file '%s' does not contain valid JSON data." % STINGINGNETTLEPY_CONFIG_FILE, exc_info=True )
			return False
			
		# The key, and the sub-dictionary 'upload', which it defines, have to exist in the Python 'Dict'.
		if 'upload' not in stingingnettlepy_config:
			#logging.error( "Could not find the key '%s' inside '%s'." % ( 'upload', STINGINGNETTLEPY_CONFIG_FILE ) )
			return False

		# The sub-dictionary has to contain the following key:value pairs:
		# {
		#	"upload": {
		#		"username": "",
		#		"password": "",
		#		"absolute_path": "",
		#		"website_root_dir": "",
		#		"host": "",
		#		"port": "",
		#

		# Make sure that all key-value pairs in the "upload" sub-dictionary exist.
		map_result = map( lambda x: x in stingingnettlepy_config["upload"], ( "username", "password", "absolute_path", "website_root_dir", "host", "port" ) )
		if not reduce( lambda x, y: x and y, map_result, True ):
			#logger.error( "The following keys have to be defined in '%s' with their key pointing to them being '%s': %s." % ( STINGINGNETTLEPY_CONFIG_FILE, "upload", ", ".join( ( "username", "password", "absolute_path", "website_root_dir", "host", "port" ) ) ) )
			return False

		# I store the sub-dict in its own variable.
		snp_upload_config = stingingnettlepy_config["upload"]


		#####################################################################
		# Check if the user has uploaded the website before.  This is done
		# by checking if the "last_version" directory already exists.		
		#####################################################################

		if os.path.exists( os.path.join( os.getcwd(), WEBSITE_OUTPUT_DIRECTORY_LAST_VERSION ) ):
			# The website has previously been uploaded to a web-server.

			#####################################################################
			# Make sure it has been uploaded to this web-server.
			#####################################################################

			try:
				ftp_client = ftputil.FTPHost( snp_upload_config["host"], snp_upload_config["username"], snp_upload_config["password"] )
			except ftputil.FTPError, e:
				print e
				return False

			if not ftp_client.path.exists( snp_upload_config["website_root_dir"] ):
				print "The directory '%s' does not exist on this server." % snp_upload_config["website_root_dir"]
				print "Please make sure that you've actually uploaded the website to this server."
				print "Otherwise, pick another server to connect to."
				return False

			#####################################################################
			# Compare the current and the last version of the website, so that
			# we know, which files have to uploaded to the remote server,
			# which files have to be deleted on the remote server,
			# which files have to be modified on the remote server,
			# and which directories have to be created or remoted on the remote
			# server.
			#
			# Comparison of directories and files is done by calculating a
			# sha256 hash.
			#####################################################################

			old_website_version = dirtree_hash_dict( WEBSITE_OUTPUT_DIRECTORY_LAST_VERSION )
			new_website_version = dirtree_hash_dict( WEBSITE_OUTPUT_DIRECTORY )

			# Here I define a Python 'Dict' in which I'll later define which directories and files have to be uploaded and/or removed from the server. 
			files_dirs_to_upload = {}
			files_dirs_to_upload["remove"] = {}
			files_dirs_to_upload["add"] = {}
			files_dirs_to_upload["remove"]["dirs"] = []
			files_dirs_to_upload["remove"]["files"] = []
			files_dirs_to_upload["add"]["dirs"] = []
			files_dirs_to_upload["add"]["files"] = []	

			# Decide which files and directories have to be uploaded, removed or modified.
			for d in old_website_version["dirs"].iteritems():
				# Do the two directories exist in both dictionaries?
				if d[0] in new_website_version["dirs"]:
					# They do.  No need to create this directory on the web-server.
					# Remove that item from 'out_dirtree_hashes'.
					new_website_version["dirs"].pop( d[0] )

				else:
					# They don't.  Remove the directory on the web-server.
					files_dirs_to_upload["remove"]["dirs"].append( d[0] )

				
			# Add the remaining directories to 'files_dirs_to_upload["add"]["dirs"]' from 'out_dirtree_hashes'.
			for x in new_website_version["dirs"].iterkeys():
				files_dirs_to_upload["add"]["dirs"].append( x )

			# Now I am going to compare files.
			for f in old_website_version["files"].iteritems():
				# Do both files exist in both dictionaries ?
				if f[0] in new_website_version["files"]:
					# They do.
					# Are they still the same? (name).  Check their checksums.
					if f[1] == new_website_version["files"][f[0]]:
						# They are still the same.  This means that this file will not be modified in any way on the web-server.
						new_website_version["files"].pop( f[0] )

					else:
						# The files do not match.  Remove the old file on the web-server.
						files_dirs_to_upload["remove"]["files"].append( f[0] )
				else:
					# The file does not exist in 'out_dirtree_hashes["files"]'.  It can be removed from the web-server.
					files_dirs_to_upload["remove"]["files"].append( f[0] )

			# Add the remaining files to 'files_dirs_to_upload["add"]["files"] from 'out_dirtree_hashes'.
			for x in new_website_version["files"].iterkeys():
				files_dirs_to_upload["add"]["files"].append( x )

			############################################################
			# And now update the entire website.
			############################################################

			snp_upload_config["website_root_dir"] = str( snp_upload_config["website_root_dir"] )

			# First remove files that have to be removed.
			for f in files_dirs_to_upload["remove"]["files"]:
				try:
					# I remove the file at the path specified.
					ftp_client.unlink( ftp_client.path.join( snp_upload_config["website_root_dir"], reduce( ftp_client.path.join, splitall( f )[1:] ) ) )
				except ftputil.error.FTPError, e:
					print e
					return False
					
			# Then remove directories.  Remove the deepest directories first, and the highest directories last.
			# For this, I first have to sort the list.  I use the 'compare_directories_depth' function which compares the amount of the path seperators of two items of the list.
			files_dirs_to_upload["remove"]["dirs"] = sorted( files_dirs_to_upload["remove"]["dirs"], cmp=compare_directories_depth )
			for d in files_dirs_to_upload["remove"]["dirs"]:
			
				# For each backslash in the path to a directory I replace the backslash with a forward slash as this is the way for web-paths.
				d_to_remove = reduce( ftp_client.path.join, splitall( d )[1:] )

				# I remove the directory at the path specified.
				try:
					ftp_client.rmdir( ftp_client.path.join( snp_upload_config["website_root_dir"], d_to_remove ) )
				except ftputil.error.FTPError, e:
					print e
					return False
			
			# Now create directories.  The highest directories are created first, while the deepest directories are created last.
			# For this task, I first have to sort and reverse the list, so that the highest directories come first.
			files_dirs_to_upload["add"]["dirs"] = reversed( sorted( files_dirs_to_upload["add"]["dirs"], cmp=compare_directories_depth ) )
			for d in files_dirs_to_upload["add"]["dirs"]:
			
				# As this is a LINUX based system, I replace every occurence of the backslash of a path with a forward slash.
				d_to_add = reduce( ftp_client.path.join, splitall( d )[1:] )

				# I call mkdir to create the directory.
				try:
					ftp_client.mkdir( ftp_client.path.join( snp_upload_config["website_root_dir"], d_to_add ) )
				except ftputil.error.FTPError, e:
					print e
					return False				
				
			# And lastly, the files are being created.
			for f in files_dirs_to_upload["add"]["files"]:

				try:
					ftp_client.upload( os.path.join( WEBSITE_OUTPUT_DIRECTORY, f[1:] ), ftp_client.path.join( snp_upload_config["website_root_dir"], reduce( ftp_client.path.join, splitall( f )[1:] ) ) )
				except ftputil.error.FTPError, e:
					print e
					return False

			
			ftp_client.close()

			############################################################
			# The website has been updated.
			# Make a copy of the current 'out/' directory and store the
			# entire content in the 'last_version/' directory.
			# Everything in the 'last_version/' directory will be
			# removed before this happens.
			############################################################

			force_copytree( WEBSITE_OUTPUT_DIRECTORY, WEBSITE_OUTPUT_DIRECTORY_LAST_VERSION )			

			return True

		else:
			# The website has never been uploaded (to any server).


			#####################################################################
			# Upload the entire website using the FTP protocol.
			# This will in future be extended to use the FTPS protocol.
			#####################################################################

			try:
				ftp_client = ftputil.FTPHost( snp_upload_config["host"], snp_upload_config["username"], snp_upload_config["password"] )
			except ftputil.FTPError, e:
				print e
				return False

			if ftp_client.path.exists( snp_upload_config["website_root_dir"] ):
				print "The directory '%s' already exists." % snp_upload_config["website_root_dir"]
				ftp_client.close()
				return False

			# Create the root-directory for the website directotires and files contained in 'out/'.
			ftp_client.mkdir( snp_upload_config["website_root_dir"] )

			# And change into it.
			#ftp_client.chdir( snp_upload_config["website_root_dir"] )

			# Now copy the entire content of the local 'out/' directory into the remote 'snp_upload_config["website_root_dir"]' directory.
			for dirpath, dirs, files in os.walk( WEBSITE_OUTPUT_DIRECTORY ):
				# Make a copy of the relative directory path.
				dirpath_copy = dirpath
				
				# Replace the root directory of dirpath in order to upload the entire website into the correct website root directory on the remote server.
				dirpath = map( str, splitall( dirpath.replace( "out", snp_upload_config["website_root_dir"], 1 ) ) )
				
				# Create directories first.
				for d in dirs:
					rel_dirpath = copy.deepcopy( dirpath )
					rel_dirpath.append( d )
					ftp_client.mkdir( ftp_client.path.join( *rel_dirpath ) )

				# Create files next.
				for f in files:
					rel_filepath = copy.deepcopy( dirpath )
					rel_filepath.append( f )
					ftp_client.upload( os.path.join( dirpath_copy, f ), ftp_client.path.join( *rel_filepath ) )

			# Close the connection.
			ftp_client.close()


			############################################################
			# The website has been uploaded for the first time.
			# Make a copy of the current 'out/' directory and store the
			# entire content in the 'last_version/' directory.
			############################################################

			force_copytree( WEBSITE_OUTPUT_DIRECTORY, WEBSITE_OUTPUT_DIRECTORY_LAST_VERSION )

			return True

		return False


	def generate( self ):
		"""Generates the website and puts the generated website into the 'out/' directory.
		
		This function generates the website and puts the generated website into the 'out/' directory.
		The same happens inside the "test" and "upload" functions.

		For more information, please read through the comments and the source-code of the function body.
		
		"""		
		# Make sure all required directories and files exist.
		dirs_exist = directories_exist( STINGINGNETTLEPY_PROJECT_WEBSITE_DIRECTORY_REQUIRED_DIRS_AND_FILES["directories"] )
		files_all_existing = files_exist( STINGINGNETTLEPY_PROJECT_WEBSITE_DIRECTORY_REQUIRED_DIRS_AND_FILES["files"] )

		if not dirs_exist or not files_all_existing:
			return False

		# Delete the "out" directory if it does exist and copy the contents of the 'in/' directory to 'out/'.  Create 'out/' if it does not exist.
		force_copytree( os.path.join( os.getcwd(), WEBSITE_INPUT_DIRECTORY ), os.path.join( os.getcwd(), WEBSITE_OUTPUT_DIRECTORY ) )
		logging.debug( "Copied 'in/' to 'out/'." )

		# Load navigation data / store the data for the top -and sub navigation in a dictionary.
		self.navigation_data = load_json( NAVIGATION_FILE, True )
		logging.debug( "Loaded navigation data." )		

		# Load templates.
		templates = self.load_templates( TEMPLATES_DIRECTORY )
		logging.debug( "Loaded templates." )
		
		# Load modules.
		modules = self.load_modules( MODULES_DIRECTORY )
		logging.debug( "Loaded modules" )
		
		# Load the global configuration.  The function returns a dictionary with key:value pairs.
		global_configuration = load_json( os.path.join( os.getcwd(), WEBSITE_OUTPUT_DIRECTORY, GLOBAL_CONFIG_FILE ) )
		logging.debug( "Loaded the global configuration." )
		
		# The global configuration has been read so the accociated file is no longer required.  Delete it.  It would only get in the way.
		os.unlink( os.path.join( os.getcwd(), WEBSITE_OUTPUT_DIRECTORY, GLOBAL_CONFIG_FILE ) )
		logging.debug( "Deleted the global configuration file." )

		# Generate the main navigation for all pages.
		global_configuration.update( { "main_navigation": self.make_top_nav() } )
		logging.debug( "Generated the main navigation for all pages." )

		# For each a file a temporary "current_config" Python Dict has to be created.  It will contain everything of the global configuraton, the local configuration and the file configuration.
		current_config = copy.deepcopy( global_configuration )
		
		# Also, we need to watch out for a local configuration file in each new directory we encounter in the "out/" directory.  In order to detect if a new directory is being walked I am going to use a temporary variable that stores the absolute dirpath of the previous iteration.
		previous_dirpath = ""

		# Walk the "out" directory and convert every ".txt" file to a ".html" file by applying templates, modules and more.  Do not alter anything inside a directory that begins with an "_" (underscore).
		logging.debug( "Generating the website." )
		for dirpath, dirnames, filenames in os.walk( os.path.join( os.getcwd(), WEBSITE_OUTPUT_DIRECTORY ) ):
		
			# Ignore all files in a directory which start with an underscore. (such as _css, _js, ...)
			# If there is a directory without an underscore inside a directory with an underscore ('_'), then these files in that directory will _not_ be ignored; they will be rendered.
			if os.path.basename( dirpath ).startswith( "_" ):
				continue
			else:
				# We are inside a directory in which text files will be rendered into HTML files.  Have we already been in this directory before?  If not, then load the "global_configuration" into the "current_config" file and then look for a local configuration file and load it.  Otherwise, simply continue.
				if previous_dirpath != dirpath:
				
					# We haven't been here before.
					previous_dirpath = dirpath					
				
					# Every file in a directory uses the global configuration.
					current_config = copy.deepcopy( global_configuration )
				
					# Does a local configuration file exist in this directory?  If yes, then load its config.  Otherwise continue.
					if os.path.exists( os.path.join( dirpath, LOCAL_CONFIG_FILE ) ):
					
						# Load the local configuration whose configuration affects all pages in the current directory.
						local_config = load_json( os.path.join( dirpath, LOCAL_CONFIG_FILE ) )

						# Delete the local configuration file as it is no longer required.
						os.unlink( os.path.join( dirpath, LOCAL_CONFIG_FILE ) )

						# Update the "current_config" dictionary with what is in the "local_config" dictionary.
						current_config.update( local_config )

				# Since I might have removed the local_configuration file, the 'filenames' list is not up-to-date anymore.  This is why I use 'os.listdir' in conjunction with 'filter' and map to extract all files that currently still exist in the current directory.
				filenames = map( os.path.basename, filter( os.path.isfile, [ os.path.join( dirpath, file_or_dir ) for file_or_dir in os.listdir( dirpath ) ] ) )

				# Now is the time to create the sub navigation.  The sub navigation is only created if any files exist in the current directory.
				if len( filenames ) > 0:
				
					# Generate the sub-navigation for this directory/sub-section, and store in the current configuration.
					current_config.update( { "sub_navigation": self.make_sub_nav( os.path.basename( dirpath ) ) } )
										
				# Render each '.txt' file in the current directory.	 (Turn .txt files into .html files.)			
				for file_to_render in filenames:

					# Read the entire content of the file to be rendered.
					file_content = read_file( os.path.join( dirpath, file_to_render ) )
					
					# Truncate the file to size 0, so it is completly empty.
					truncate_file( os.path.join( dirpath, file_to_render ) )

					# Make a copy of the local configuration for later use.
					current_config_copy = copy.deepcopy( current_config )

					# Extract the configuration and the containers.
					
					# A '.txt' file is made up of either:
					# 	- a configuration and a containers part
					#	- a configuration part
					#	- a containers part.

					# Extract the configuration of this file.					
					file_configuration = self.extract_config( file_content )

					# At this point, 'current_config' contains the configuration defined throught the global- and the local configuration.
					if file_configuration is not None:
					
						# 'current_config' will now contain the configuration defined through the global, the local and the file configuration.
						current_config.update( file_configuration )

					# Execute any functions that have been defined in the configuration, and store the result in 'current_config' as new key:value pairs.
					current_config.update( self.execute_functions( current_config ) )

					# Extract any containers.
					file_containers = self.extract_containers( file_content )
											
					if file_containers == False:
						print "An error has occured while extracting containers from '%s'" % os.path.join( dirpath, file_to_render )
						return False
					elif file_containers is not None:

						# Every single container is a template aswell.  This allows the user to define placeholders for a Python 'string.Template' instance inside a container.  It is meant to be used for functions that a developer wishes to include, for example a function that generates the HTML code for an image-gallery.
						# New in Stingingnettlepy v0.2.

						# Create a deep copy of the dictionary 'file_containers'.
						copy_of_file_containers = copy.deepcopy( file_containers )

						# Walk each container.
						for f_container in copy_of_file_containers.iteritems():

							# Create a new instance of the 'string.Template' class using the value of each dictionary key:value pair as the string for the call to 'string.Template'.
							f_container_template = string.Template( f_container[1] )

							# Apply 'current_config' on this template and store the rendered string in 'file_containers[f_container[0]]'.
							# That means that the content that previously existed is overwritten.
							file_containers[f_container[0]] = f_container_template.safe_substitute( current_config )

						# Update 'current_config' with the key:value pairs of 'file_containers'.
						current_config.update( file_containers )
						
					else:

						# Do nothing.  This has to be removed somehow, as it is ugly.
						pass
					

					# Pick the correct template for rendering the '.txt' file to a '.html' file.  A template key:value pair HAS to exist in 'current_config'.
					if not "template" in current_config:
						print "Error:  You did not provide a template."
						return False
					else:
						# Create an instance of the 'Template' class which can be found in the 'string' module.
						used_template = string.Template( templates[current_config["template"]] )

						# The key:value pair for the template is no longer required, and would only get in the way.  Delete it.
						current_config.pop( "template" )
						
						# Now generate the final html code for this file.  Perform the substitution.
						generated_html = used_template.safe_substitute( current_config )

						# Store the generated HTML code in the same file and rename the file to '*.html'.
						write_file( os.path.join( dirpath, file_to_render ), generated_html )
						os.rename( os.path.join( dirpath, file_to_render ), os.path.join( dirpath, file_to_render ).replace( ".txt", ".html" ) )

					# For a new file, we need to use a new file configuration.
					# 'current_config_copy' contains the global- and the local configuration.
					current_config = copy.deepcopy( current_config_copy )

				# For a new file in another sub directory, we need to use a new file configuration.
				# Please note that 'current_config' will be filled with what is inside the global configuration at the top of the loop body.
				current_config = {}				
				

	def load_templates( self, templates_root_dir ):
		"""Reads the content of all all templates found in 'templates_root_dir' and stores the name of the template and its content as a key-value pair inside a returned dictionary.
		
		This function reads the content of all all templates found in 'templates_root_dir' and stores the name of the template and its content as a key-value pair inside a returned dictionary.
		This dictionary is then returned to the calling function.

		Args:
			templates_root_dir: The name of the directory where the templates for a website are stored.

		Returns:
			A Python 'dict'.
			
		"""
		template_data = {}
		for tmp in os.listdir( os.path.join( os.getcwd(), templates_root_dir ) ):
		
			# Open the template-file in reading mode.  Read the content of the template as a string and store it temporarily.
			content = read_file( os.path.join( os.getcwd(), templates_root_dir, tmp ) )

			# Store the content in a dictionary where the 'key' is the filename of the template.
			template_data[tmp] = content

		return template_data


	def load_modules( self, modules_root_dir ):
		"""Loads the modules found in the 'modules_root_dir' into the local namespace of this class.
		
		This function loads the modules found in the 'modules_root_dir' into the local namespace of this class.

		Args:
			modules_root_dir: The name of the directory where the modules for a website can be found.
			
		"""
		sys.path.append( os.getcwd() )

		for module_file in os.listdir( os.path.join( os.getcwd(), modules_root_dir ) ):
			if module_file.startswith( "__" ):
				continue
			else:
				importlib.import_module( modules_root_dir + "." + os.path.splitext( module_file )[0] )


	def extract_config( self, content ):
		"""Searches and extracts the entire configuration if a configuration can be found.
		
		This function searches and extracts the entire configuration if a configuration can be found.
		The configuration has to be in JSON.
		If a configuration can be found, then a Python 'Dict' object, containing its keys and values, is created and returned.
		Otherwise, None is returned.

		Args:
			content: A string that contains the content of a text-file.

		Returns:
			A Python 'dict' or 'None'.
			
		"""
		# Read the content line-by-line until the delimiter '---config---' is found.
		temp_content = ""
		config_delimiter_found = False
		
		for line in content.splitlines( True ):
			if line.find( "---config---", 0 ) is not -1:
				config_delimiter_found = True
				break
			else:
				temp_content += line

		# Make sure that '---config---' was not on the first line.
		if len( temp_content.splitlines() ) == 0 or not config_delimiter_found :
			return None
		else:
			# Return a Python 'Dict'.
			return json.loads( temp_content )


	def extract_containers( self, content ):
		"""This function extracts any placeholers that can be found and returns a Python 'Dict', where the 'key' is the name of the 'placeholder', and its 'value' the content of the placeholder.

		In the following lines, I refer to 'placeholder' by the name of 'containers'.
		
		This function extracts one or more containers of 'content' if any can be found.
		A container will be stored as a key:value pair inside a Python 'Dict', where the 'key' is the name or identifier of the container (without the leading '|||'(pipes)), while the 'value' is the text/HTML code between the identifiers/delimiters of the container.
		If no containers can be found, 'None' is returned.
		If an error occurs (such as a forgotten closing delimiter), 'False' is returned.
		Otherwise the Python 'Dict' is returned.

		Args:
			content: A string that contains the content of a text-file.

		Returns:
			A Python 'dict', 'False' or 'None'.

		"""
		# In this code fragment, I try to find out on which line the substring '---config---' is in the content.
		# If it is not found, I continue reading the next line until the last line.
		# If it is not found on the last line, I return 'None'.
		# If it is found on the last line, I return 'None'.
		# If it is found on any other line, I slice the list 'lines' and copy every line from where '---config---' + 1 (the next line) has been foundline to the end.
		container_content = []
		lines = content.splitlines( True )
		for x in xrange( len( lines ) ):
			if lines[x].find( "---config---", 0 ) is not -1:
				if len( lines ) is not ( x + 1 ):
					container_content = lines[x+1:]
					break
				else:
					#return None
					pass
			else:
				container_content

		if len( container_content ) == 0 :
			#return None
			container_content = lines

		# Read from the next line from where '---config---' has been found in 'content'.
		# In this code fragment I am going to extract any containers and their content, where 'content' is text/HTML code.
		# For this task I first define 5 variables:
		# This Python 'Dict' will contain any containers and their content.  The name of the container is the start-tag/end-tag in the file read without the leading '|||' pipe characters.
		file_containers = {}
		# This variable is set to 'True' if the beginnning start-tag has been found.
		container_start_found = False
		# This variable is set to 'True' if the ending end-tag has been found.
		container_end_found = False
		# In this variable I am going to store the complete name of the container-name (that is 'start-tag'/'end-tag') that has been found.
		container_name = ""
		# In this variable I am going to store the content (lines) that are contained between the 'start-tag' and the 'end-tag'.  I am simply going to append the next line to this variable.
		single_container_content = u""

		# This is the actual code-fragment which I use to extract the containers and their content.

		# I read each line.
		for line in container_content:
			# Has a start-tag already been found ?
			if not container_start_found:
				# A start-tag has not yet been found.
				# I use a regular expression and the function 're.match' to check if a start-tag that describes the beginning of a container has been found.
				# I use 're.match' because the start-tag of the container has to begin on the very first character of the line.
				match_object = re.match( "\|\|\|[a-zA-Z_]+", line )
				# 'match_object' is what is returned by 're.match'  It is set to 'True' if a start-tag has been found.  Otherwise it is set to 'False'.
				if match_object:
					# A start-tag has beeen found.
					container_start_found = True
					container_end_found = False
					# I store the _complete_ name of this start-tag in 'container_name'.
					container_name = match_object.group(0)
					# Then I simply jump to the top of the for-loop to read the next line.
					continue
					
			# A start-tag has already been found
			else:
				# I check if the corresponding end-tag has been found on the current line.
				if line.find( container_name, 0 ) is not -1:
					# It has been found.
					container_end_found = True
					container_start_found = False
					# I add the newly found container to the 'file_containers' Python 'Dict'.
					# The 'key' is the name of the container without the leading '|||' pipe characters.
					# The 'value' is the content that has been found between the start-tag and the end-tag of a container.
					# To remove the leading '|||' characters I use what is called 'slicing' in Python.
					# 'container_name[3:]' means that I want a new string without the first three characters of this string, which are no doubt '|||'.
					file_containers[container_name[3:]] = single_container_content
					container_name = ""
					single_container_content = ""

				else:
					# The correspondig end-tag has not been found on this line.
					# This just means that this is actual text/HTML code.
					# I append this line.
					single_container_content += line.decode( "utf-8" )

		# After being done extracting containers, I need to make sure that no error has occured, such as forgotton end-tags.
		if container_start_found and not container_end_found:
			print "Could not find the closing container tag for '%s'" % container_name
			return False

		# I check if any containers have been found at all.		
		if len( file_containers ) >= 1:
			# Return the Python 'Dict' that contains the containers of this file.
			return file_containers
		else:
			# No containers have been found.
			return None


	def execute_functions( self, current_config ):
		"""This function executes any functions defined in 'current_config', updates the dictionary 'current_config' afterwards, and returns the dictionary.

		For more information, please read though the body of the function and its comments.

		Args:
			current_config: A dictionary that contains the current configuration for a file, and to be used on a file.

		Returns:
			A dictionary that represents the final configuration for a TTH file.
			
		"""
		# First I make a deep copy of the Python 'Dict' 'current_config', as I might need to remove items from it while iterating over it.
		# This solely depends on whether there are any functions to execute or not.
		current_config_copy = copy.deepcopy( current_config )

		# I go through each key:value pair of the dictionary.
		for item in current_config_copy.iteritems():
			# I only need to watch out for 'keys' that begin with a "func_" substring.
			if item[0].startswith( "func_"):
			
				# I execute the function based on the values that are defined inside this specific sub-dict.
				# First I store the values that are stored inside this sub-dict.
				module_name = item[1]["module_name"]
				function_name = item[1]["function_name"]
				function_parameters = item[1]["parameters"]
				container_name = item[1]["container_name"]

				# Now that I have all the information required, I can safely remove the key and the associated sub-dict from 'current_config'.
				current_config.pop( item[0] )

				# Now I execute the function and store the return value of the function as a new key:value pair inside the 'current_config' Python 'Dict'.
				# What kind of parameter is passed?
				if type( function_parameters ) == dict:
					# It is a dictionary.
					current_config[container_name] = getattr( sys.modules[MODULES_DIRECTORY + "." + module_name], function_name )( **function_parameters )
				elif type( function_parameters ) == list or type( function_parameters ) == tuple:
					# It is either a list or a tuple.
					current_config[container_name] = getattr( sys.modules[MODULES_DIRECTORY + "." + module_name], function_name )( *function_parameters )				
				else:
					# Either a string, an int, a float or a boolean is passed.
					current_config[container_name] = getattr( sys.modules[MODULES_DIRECTORY + "." + module_name], function_name )( function_parameters )
				
		# I return this new dictionary (current_config).
		return current_config


	def make_top_nav( self ):
		"""Generates the HTML for the top navigation for all pages.

		Returns:
			A string 'main_nav' that contains the HTML code for the main navigation.
			
		"""
		main_nav = "<ul>"
		for x in self.navigation_data:
			# Split the array into sub-arrays consisting of 2 values, e.g. [ 1, 2, 3, 4, 5, 6 ] => [[ 1, 2 ], [ 3, 4 ], [ 5, 6 ]]
			# The first value represents the filename of the file to link to; the second is not important, as it isn't used.
			main_nav_data = [self.navigation_data[x][y:y+2] for y in range( 0, len( self.navigation_data[x] ), 2 )]
			
			for item in main_nav_data:
				# For the main navigation, we only need to find the 'index.html' file.
				# It should exist only once.
				if item[0] == "index.html":
					main_nav += "<li><a href='%s'>%s</a></li>" % ( posixpath.join( "/", self.website_name, x, item[0] ), x.title().replace( "_", " " ) )
		main_nav += "</ul>"

		return main_nav


	def make_sub_nav( self, sub_directory ):
		"""Generates the HTML for the sub navigation for pages in a certain sub-section/sub-directory.

		Args:
			sub_directory: A string that contains the name of the subdirectory for which the sub-navigation should be created.

		Returns:
			A string 'sub_nav' that contains the HTML code for the sub navigation.
			
		"""
		if not self.navigation_data.has_key( sub_directory ):
			return ""

		# Split the array into sub-arrays consisting of 2 values, e.g. [ 1, 2, 3, 4, 5, 6 ] => [[ 1, 2 ], [ 3, 4 ], [ 5, 6 ]]
		# The first value represents the filename of the file to link to; the second value represents the value to be used between
		# the <a></a> tag.
		sub_nav_data = [self.navigation_data[sub_directory][y:y+2] for y in range( 0, len( self.navigation_data[sub_directory] ), 2 )]

		sub_nav = "<ul>"
		for x in sub_nav_data:
			sub_nav += "<li><a href='%s'>%s</a></li>" % ( posixpath.join( "/", self.website_name, sub_directory, x[0] ), x[1] )
		sub_nav += "</ul>"

		return sub_nav


	def make_backup( self ):
		"""Creates a backup of the entire 'SWPD' directory without the 'backups/' directory of course,
		and stores the backup in the 'backups/' directory, using the date (YYYY-MM-DD)
		and the current time (HH-MM-SS) as the name as the name for the newly created backup directory.

		The name of the backup direcotry looks like this:
			2013-12-23-17-11-50
		"""

		# Make sure there is a 'backups/' directory.
		if not os.path.exists( WEBSITE_BACKUPS_DIRECTORY ):
			print "The '%s' directory does not exists.  Please create it manually." % WEBSITE_BACKUPS_DIRECTORY
			return False

		# Generate the character sequence of the new backup directory for the new backup.
		current_timestamp = time.localtime( time.time() )
		backup_name = "%s-%s-%s-%s-%s-%s" % ( current_timestamp.tm_year, current_timestamp.tm_mon, current_timestamp.tm_mday, current_timestamp.tm_hour, current_timestamp.tm_min, current_timestamp.tm_sec )

		# Make sure the directory does not yet exist in the 'backups/' directory.
		if not os.path.exists( os.path.join( WEBSITE_BACKUPS_DIRECTORY, backup_name ) ):
			# The directory does not exist.  Perform the backup.
			
			# First, create the backup directory.
			os.mkdir( os.path.join( WEBSITE_BACKUPS_DIRECTORY, backup_name ) )

			# Define which directories and top-level-files should be included in the backup.
			dirs_to_backup = ( WEBSITE_INPUT_DIRECTORY, TEMPLATES_DIRECTORY, MODULES_DIRECTORY )
			top_level_files_to_backup = ( STINGINGNETTLEPY_CONFIG_FILE, NAVIGATION_FILE )

			# Perform the backup of the directories, including any sub-directories and files.
			for x in dirs_to_backup:
				shutil.copytree( x, os.path.join( WEBSITE_BACKUPS_DIRECTORY, backup_name, x ) )

			# Perform the backup of single top-level-files.
			for x in top_level_files_to_backup:
				shutil.copyfile( os.path.join( os.getcwd(), x ), os.path.join( WEBSITE_BACKUPS_DIRECTORY, backup_name, x ) )

			# Backup finished.
			return backup_name
		else:
			print "The backup directory '%s' already exists." % os.path.join( WEBSITE_BACKUPS_DIRECTORY, backup_name )
			return False 
